# Read in the data
rawTrain <- fread(file = params$trainPath)[, Partition := "Train"] %>%
  rename(FirstFloorSF = "1stFlrSF", SecondFloorSF = "2ndFlrSF", ThreeSeasonPorch = "3SsnPorch")
rawTest <- fread(file = params$testPath)[, SalePrice := NA][, Partition := "Test"] %>%
  rename(FirstFloorSF = "1stFlrSF", SecondFloorSF = "2ndFlrSF", ThreeSeasonPorch = "3SsnPorch")
rawData <- rbindlist(list(rawTrain, rawTest), use.names = TRUE)
colClasses <- sapply(rawData, class) %>% set_names(colnames(rawData))
# Create a DT::datatable for printing to HTML
make_dt_hpc <- function(dataSet, pageLength = 10) {
  
  # Determine whether or not knitting is in progress
  knitting <- isTRUE(getOption('knitr.in.progress'))
  
  # Create the DT::datatable object
  dt <- DT::datatable(
    data = dataSet
    , filter = "top"
    , class = "nowrap"
    , escape = FALSE
    , width = if (knitting) "100%" else NULL
    , height = if (knitting) "auto" else NULL
    , options = list(scrollX = TRUE, pageLength = pageLength)
  )
  
  # Return the DT::datatable object
  return(dt)
  
}
# Print the section title
raw_html_header("House Prices Kaggle Competition - EDA", headerLevel = 1, details = FALSE)

House Prices Kaggle Competition - EDA

# Loop over most of the columns in the raw data, creating tables and graphs
colData <- sapply(
  setdiff(colnames(rawData), c("Id", "SalePrice", "Partition")) %>% sort()
  # "LotFrontage"
  , simplify = FALSE
  , function(colName) {
    
    # Print a header for the column
    raw_html_header(tools::toTitleCase(colName), headerLevel = 2, open = openDetails)
    
    # Extract the columns
    fullCol <- rawData[[colName]]
    trainCol <- rawTrain[[colName]]
    testCol <- rawTest[[colName]]
    
    # Get the column class, & info on missing values
    colClass <- class(fullCol)
    numMissingFull <- KO::num_missing(fullCol)
    numMissingTrain <- KO::num_missing(trainCol)
    numMissingTest <- KO::num_missing(testCol)
    percMissingFull <- KO::percent_missing(fullCol)
    percMissingTrain <- KO::percent_missing(trainCol)
    percMissingTest <- KO::percent_missing(testCol)
    
    # Get the number of distinct values, & summaries for numeric columns
    numDistinctFull <- n_distinct(fullCol)
    numDistinctTrain <- n_distinct(trainCol)
    numDistinctTest <- n_distinct(testCol)
    colSummaryFull <- summary(fullCol)
    colSummaryTrain <- summary(trainCol)
    colSummaryTest <- summary(testCol)
    
    # Determine whether or not knitting is in progress (used to choose how to print certain)
    knitting <- isTRUE(getOption('knitr.in.progress'))
    
    
    # Create an info table
    infoData <- data.table(
        Dataset = c("Full", "Train", "Test")
        , "Number Missing" = c(numMissingFull, numMissingTrain, numMissingTest)
        , "Percent Missing" = c(percMissingFull, percMissingTrain, percMissingTest)
        , Summary = list(colSummaryFull, colSummaryTrain, colSummaryTest) %>%
            sapply(function(x) paste0(names(x), ": ", x, collapse = " | "))
      )
    infoTable <- make_dt_hpc(infoData) %>%
      DT::formatPercentage(., columns = c("Percent Missing"), digits = 2)
    
    # Print the info table
    raw_html_header("Column Info", headerLevel = 3)
    if (knitting) cat(knitr::knit_print(infoTable)) else print(infoTable)
    cat("</details>")
    
    
    # Set the number of bins to use
    numBins <- ifelse(numDistinctTrain > 20, 10, numDistinctTrain)
    
    # Create the quantile-spaced data & plot
    quantileData <- KO::binned_one_way_data(x = trainCol, yData = rawTrain[, .(SalePrice)], bins = numBins) %>%
      setnames(old = c("Bins__", "SalePrice", "Weight__"), new = c("Bins", "Sale Price", "Weight"))
    quantileTable <- make_dt_hpc(quantileData, pageLength = numBins) %>%
      DT::formatPercentage(., columns = "Weight", digits = 2) %>%
      DT::formatRound(., columns = "Sale Price", digits = 0)
    quantilePlot <- KO::binned_one_way_plot(x = trainCol, yData = rawTrain[, .(SalePrice)],
      xlab = tools::toTitleCase(colName), ylab = "Sale Price", bins = numBins, plotly = params$plotly)
    
    # Create the equally-spaced data & plot
    equalData <- KO::binned_one_way_data(x = trainCol, yData = rawTrain[, .(SalePrice)], type = "equal", bins = numBins) %>%
      setnames(old = c("Bins__", "SalePrice", "Weight__"), new = c("Bins", "Sale Price", "Weight"))
    equalTable <- make_dt_hpc(equalData, pageLength = numBins) %>%
      DT::formatPercentage(., columns = "Weight", digits = 2) %>%
      DT::formatRound(., columns = "Sale Price", digits = 0)
    equalPlot <- KO::binned_one_way_plot(x = trainCol, yData = rawTrain[, .(SalePrice)], type = "equal",
      xlab = tools::toTitleCase(colName), ylab = "Sale Price", bins = numBins, plotly = params$plotly)
    
    
    # Show the quantile data
    raw_html_header("Quantile-Spaced Data", headerLevel = 3)
    if (knitting) cat(knitr::knit_print(quantileTable)) else print(quantileTable)
    cat("</details>")
    
    # Show the quantile plot
    raw_html_header("Quantile-Spaced Plot", headerLevel = 3)
    print(if (params$plotly) htmltools::tagList(quantilePlot) else quantilePlot)
    cat("</details>")
    
    # Show the equally spaced data & plot, if needed
    if (numDistinctTrain > 20) {
      
      # Show the equally spaced data
      raw_html_header("Equally-Spaced Data", headerLevel = 3)
      if (knitting) cat(knitr::knit_print(equalTable)) else print(equalTable)
      cat("</details>")
      
      # Show the equal spaced plot
      raw_html_header("Equally-Spaced Plot", headerLevel = 3)
      print(if (params$plotly) htmltools::tagList(equalPlot) else equalPlot)
      cat("</details>")
      
    }
    
    
    # Close the details section, & return the info
    cat("</details>")
    return(dplyr::lst(
      colClass, numMissingFull, numMissingTrain, numMissingTest,
      percMissingFull, percMissingTrain, percMissingTest,
      colSummaryFull, colSummaryTrain, colSummaryTest,
      quantileData, quantilePlot, equalData, equalPlot,
      infoTable, quantileTable, equalTable
    ))
    
  }
  
)

Alley

Alley

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

BedroomAbvGr

BedroomAbvGr

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

BldgType

BldgType

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

BsmtCond

BsmtCond

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

BsmtExposure

BsmtExposure

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

BsmtFinSF1

BsmtFinSF1

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

BsmtFinSF2

BsmtFinSF2

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

BsmtFinType1

BsmtFinType1

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

BsmtFinType2

BsmtFinType2

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

BsmtFullBath

BsmtFullBath

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

BsmtHalfBath

BsmtHalfBath

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

BsmtQual

BsmtQual

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

BsmtUnfSF

BsmtUnfSF

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

CentralAir

CentralAir

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Condition1

Condition1

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Condition2

Condition2

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Electrical

Electrical

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

EnclosedPorch

EnclosedPorch

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

ExterCond

ExterCond

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Exterior1st

Exterior1st

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Exterior2nd

Exterior2nd

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

ExterQual

ExterQual

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Fence

Fence

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

FireplaceQu

FireplaceQu

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Fireplaces

Fireplaces

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

FirstFloorSF

FirstFloorSF

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

Foundation

Foundation

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

FullBath

FullBath

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Functional

Functional

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

GarageArea

GarageArea

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

GarageCars

GarageCars

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

GarageCond

GarageCond

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

GarageFinish

GarageFinish

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

GarageQual

GarageQual

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

GarageType

GarageType

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

GarageYrBlt

GarageYrBlt

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

GrLivArea

GrLivArea

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

HalfBath

HalfBath

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Heating

Heating

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

HeatingQC

HeatingQC

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

HouseStyle

HouseStyle

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

KitchenAbvGr

KitchenAbvGr

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

KitchenQual

KitchenQual

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

LandContour

LandContour

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

LandSlope

LandSlope

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

LotArea

LotArea

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

LotConfig

LotConfig

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

LotFrontage

LotFrontage

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

LotShape

LotShape

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

LowQualFinSF

LowQualFinSF

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

MasVnrArea

MasVnrArea

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

MasVnrType

MasVnrType

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

MiscFeature

MiscFeature

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

MiscVal

MiscVal

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

MoSold

MoSold

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

MSSubClass

MSSubClass

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

MSZoning

MSZoning

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Neighborhood

Neighborhood

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

OpenPorchSF

OpenPorchSF

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

OverallCond

OverallCond

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

OverallQual

OverallQual

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

PavedDrive

PavedDrive

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

PoolArea

PoolArea

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

PoolQC

PoolQC

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

RoofMatl

RoofMatl

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

RoofStyle

RoofStyle

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

SaleCondition

SaleCondition

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

SaleType

SaleType

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

ScreenPorch

ScreenPorch

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

SecondFloorSF

SecondFloorSF

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

Street

Street

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

ThreeSeasonPorch

ThreeSeasonPorch

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

TotalBsmtSF

TotalBsmtSF

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

TotRmsAbvGrd

TotRmsAbvGrd

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Utilities

Utilities

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

WoodDeckSF

WoodDeckSF

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

YearBuilt

YearBuilt

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

YearRemodAdd

YearRemodAdd

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot

Equally-Spaced Data

Equally-Spaced Data

Equally-Spaced Plot

Equally-Spaced Plot

YrSold

YrSold

Column Info

Column Info

Quantile-Spaced Data

Quantile-Spaced Data

Quantile-Spaced Plot

Quantile-Spaced Plot